package main

import "strings"
import "sort"
import "net/http"
import "net/url"
import "github.com/golang/glog"
import "github.com/PuerkitoBio/goquery"
import "github.com/PuerkitoBio/purell"

// Returns the content type stripped of any additional parameters (following the ;).
func getContentType(res *http.Response) string {
	val := res.Header.Get("Content-Type")
	args := strings.SplitN(val, ";", 2)
	if len(args) > 0 {
		return strings.TrimSpace(args[0])
	}
	return val
}

func getUniqueSortedStrings(strs []string, ustrs []string) []string {
	if len(strs) == 0 {
		return ustrs
	}

	sort.Strings(strs)
	ustrs = append(ustrs, strs[0])
	for i := 1; i < len(strs); i++ {
		if strs[i] != strs[i-1] {
			ustrs = append(ustrs, strs[i])
		}
	}
	return ustrs
}

// Return all links found in the doc, interpreted in the context
// of ctxurl.
// The links are normalized using purell.
// Links that normalize to ctxurl itself are not returned.
// Links are returned unsorted and may include duplicates.
func getLinks(ctxurl *url.URL, doc *goquery.Document) []string {
	ctxurlstr := ctxurl.String()
	pflags := purell.FlagsUsuallySafeGreedy | purell.FlagRemoveFragment

	links := make([]string, 0, 50)
	doc.Find("a[href]").Each(func(i int, s *goquery.Selection) {
		val, _ := s.Attr("href")
		// Resolve address relative to context address
		u, err := ctxurl.Parse(val)
		if err != nil {
			glog.Errorf("resolve URL '%s': %v\n", val, err)
			return
		}
		if u.Host != ctxurl.Host {
			glog.V(2).Infof("discarding URL=[%s] for other host", u)
			return
		}
		nurl := purell.NormalizeURL(u, pflags)
		if nurl == ctxurlstr {
			return // don't bother tracking self-references
		}
		links = append(links, nurl)
	})
	return links
}
